# -*- coding: UTF-8 -*-
from pyspark.sql import SparkSession
from pyspark.sql.types import *


class SparkTextFileLoader(object):
    spark = None
    SQLScheme = None
    DataFrame = None

    def __init__(self, conf):
        self.conf = conf


    def getSparkSession(self):
        """获取sparkSession的实例"""
        self.spark = SparkSession \
            .builder \
            .master(self.conf["sparkMaster"]) \
            .appName(self.conf["appName"]) \
            .config("spark.sql.warehouse.dir",self.conf["sparkWarehousePath"]) \
            .config("spark.sql.shuffle.partitions",6) \
            .config("spark.work.memory","16g") \
            .config("spark.work.cores",12) \
            .getOrCreate()
        return self.spark



    def dbLoad(self):
        """执行数据导入"""
        fileType = self.conf['fileType']
        if fileType == 'csv':
            self.DataFrame = self.spark.read.csv(self.conf['filePath'])
        if fileType == 'text':
            self.DataFrame = self.spark.read.text(self.conf['filePath'])
        return self

    def dbCache(self):
        self.DataFrame.cache()
        return self
